package com.spider.service;

import java.util.List;

import javax.management.JMException;

import com.spider.dao.HBaseDAOImp;
import com.spider.dao.HBaseDao;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.monitor.SpiderMonitor;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.scheduler.FileCacheQueueScheduler;

public class QuesPageProcesschem implements PageProcessor{

	private Site site = Site.me().setRetryTimes(3).setSleepTime(1000).setTimeOut(10000);
	private HBaseDao dao = new HBaseDAOImp();
	private int index = 0;
	private int index1 = 0;
	@Override
	public Site getSite() {
		// TODO Auto-generated method stub
		return site;
	}
	
	@Override
	public void process(Page page) {
		System.out.println("1234");
		// TODO Auto-generated method stub
		page.putField("html", page.getHtml());
		
		String html = page.getResultItems().get("html").toString();
		System.out.println(html);
			
		List<String> urls = page.getHtml()
				.xpath("//div[@class='laypage_main laypageskin_default']")
				.links().regex(".*hdSearch=.*").all();
		List<String> urlz = page.getHtml()
				.xpath("//dl[@class='z-open']")
				.links().regex(".*kid=.*").all();	
		List<String> urlBook = page.getHtml()
				.xpath("//div[@id=filter-book]")
				.links().regex(".*bid=.*").all();
		List<String> urlType = page.getHtml()
				.xpath("//div[@id=fitler-tixing]")
				.links().regex(".*qtid=.*").all();
		List<String> urlVersion = page.getHtml()
				.xpath("//div[@id=filter-version]")
				.links().regex(".*vid=.*").all();
		
		dao.insert("htmlBio", "content"+(index1++)+"", "base", "content", html);
		
		page.addTargetRequests(urls);
		page.addTargetRequests(urlz);
		page.addTargetRequests(urlBook);
		page.addTargetRequests(urlType);
		page.addTargetRequests(urlVersion);
	}
	

	public static void main(String[] args) {
		System.out.println("12345");

		Spider spider = Spider.create(new QuesPageProcesschem())
		.setScheduler(new FileCacheQueueScheduler("./chemUrlManager"))
		.addUrl("http://www.tiku.com/testPaper.html?sct=0&cn=%E7%94%9F%E7%89%A9&st=2&cid=500013&bid=800088&vid=800015&uid=800354&cptid=701132&kid=703576&qtid=600057&difficulty=&sort=0")
		.thread(5);
		
		try {
			SpiderMonitor.instance().register(spider);
		} catch (JMException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		
		spider.run();
	}
}
